# encoding: utf-8

import os

import pandas as pd
from config import config
from crawl.ak_share import AkShare
from core.common import get_data_dict


def get_scrawl_dict():
    """爬虫类型"""
    ak_share = AkShare()
    return {
        'ak_share': ak_share,
    }


def crawl_data():
    """获取数据"""
    scrawl_dict = get_scrawl_dict()
    data_list = get_data_dict()

    for item in data_list:
        if not item.get('is_need'):
            continue
        print(item.get('code'))
        original_path = os.path.join(config.ORIGINAL_PATH, item.get('cate_type'))
        os.makedirs(original_path, exist_ok=True)
        file_path = os.path.join(original_path, f'{item.get("code")}_{item.get("describe")}.xlsx')
        func = getattr(scrawl_dict.get(item.get('scrawl')), item.get('scrawl_method'))
        try:
            if os.path.isfile(file_path):
                df = pd.read_excel(file_path)
                data_df = func(item.get('code'), 'no_first')
                # 统一时间格式
                df['净值日期'] = pd.to_datetime(df['净值日期'])
                data_df['净值日期'] = pd.to_datetime(data_df['净值日期'])
                merged_df = pd.concat([df, data_df]).drop_duplicates()
                merged_df.to_excel(file_path, index=False)
            else:
                data_df = func(item.get('code'), 'first')
                data_df.to_excel(file_path, index=False)
        except:
            print(item.get('code'), '失败')
